package io.sqrtqiezi.spark.yelp

import org.apache.spark.sql.SparkSession

object Cubing {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .appName("BI-Yelp")
      .master("local[*]")
      .getOrCreate()

    val business = spark.read.format("json")
      .load("/Users/njin/bigdata/data/yelp_data/yelp_academic_dataset_business.json")
    val review = spark.read.format("json")
      .load("/Users/njin/bigdata/data/yelp_data/yelp_academic_dataset_review.json")

    business.createOrReplaceTempView("business")
    review.createOrReplaceTempView("review")

//    business.printSchema()
//    review.printSchema()

    val sqlStr = "SELECT " +
      "r.cool, r.funny, r.stars, r.useful, " +
      "l.city, l.state, r.date " +
      "FROM business l " +
      "JOIN review r ON l.business_id = r.business_id"

    println(business.rdd.getNumPartitions)
    println(review.rdd.getNumPartitions)

    val df = spark.sql(sqlStr)
    df.coalesce(1).write
      .parquet("/Users/njin/bigdata/data/yelp_dm")
//      .orc("/Users/njin/bigdata/data/yelp_dm")

//    println(df.rdd.getNumPartitions)
//      .write
//      .orc("/Users/njin/bigdata/data/yelp_dm")

    Thread.sleep(1000000)
    spark.stop()
  }
}
